#coding:utf-8

import re,hashlib,time,urllib,httplib
from middle.bloomFilterSpider import bloomFilterSpider

class BiyingNewsSpider(bloomFilterSpider):
    """
必应新闻搜索关键字
use case :
a = BiyingNewsSpider("biying")
a.start("四川大学",1)
    """
    def start(self,key,page):
        self.key = key
        key = {'q':key}
        for i in range(page):
            self.run('http://cn.bing.com/news/search?'+urllib.urlencode(key)+'&first='+str(i*10+1))
    def extract(self,html):
        # url,pubtime,title,content,searchword,md5
        for sel in html.xpath('//div[contains(@class,"sn_r")]'):
            a = sel.xpath('div[@class="newstitle"]/a')
            span = sel.xpath('div[@class="sn_txt"]/div[@class="sn_oi"]/span[@class="sn_ST"]')
            if a and span:
                a=a[0]
                span = span[0]
                res = {}
                res["url"] = a.attrib.get("href","")
                if self.existUrl(res["url"]):
                    continue;
                res["pubtime"] = self.convertTime(span.xpath('string(span[@class="sn_tm"]/text())'))
                res["site"] = span.xpath('string(cite[@class="sn_src"]/text())').strip().encode("utf8")
                res["title"] = self.getInnerText(a)
                res["content"] = self.opener.openurl(res["url"])
                res["md5"] = hashlib.md5(res["content"]).hexdigest()
                res["searchword"] = self.key
                yield res
    def convertTime(self,s):
        m = re.match(u"(\d+) 分",s)
        if m:
            return int(time.time() - int(m.group(1))*60)
        m = re.match(u"(\d+) 小时",s)
        if m:
            return int(time.time() - int(m.group(1))*3600)
        m = re.match(u"(\d+) 天",s)
        if m:
            return int(time.time() - int(m.group(1))*86400)
        m = re.match("\d+/\d+/\d",s)
        if m:
            return int(time.mktime(time.strptime(s,"%Y/%m/%d")))
        return "0"